
# import libraries

import pandas as pd
import plotly.express as px
from datetime import datetime, timedelta
import seaborn as sns
from matplotlib import pyplot as plt
import numpy as np
from scipy import stats as st
%matplotlib inline
import plotly.graph_objects as go
import math as mth
from statsmodels.stats.proportion import proportions_ztest
import warnings
warnings.filterwarnings('ignore')


#function for primary analysis of the dataset

def dataset (dataset):
    dataset.columns = [x.lower().replace(' ', '_') for x in dataset.columns.values]
    display(dataset.info())
    print('*'*50)
    display(dataset.describe())
    print('*'*50)
    display(dataset.sample(10))
    print('*'*50)
    display('Number of dublicate values', dataset.duplicated().sum())


#uploading data

events = pd.read_csv('/datasets/final_ab_events.csv', parse_dates=['event_dt'])
marketing_events = pd.read_csv('/datasets/ab_project_marketing_events.csv', parse_dates=['start_dt', 'finish_dt'])
new_users = pd.read_csv('/datasets/final_ab_new_users.csv', parse_dates=['first_date'])
participants = pd.read_csv('/datasets/final_ab_participants.csv')

dataset(events)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 440317 entries, 0 to 440316
Data columns (total 4 columns):
 #   Column      Non-Null Count   Dtype         
---  ------      --------------   -----         
 0   user_id     440317 non-null  object        
 1   event_dt    440317 non-null  datetime64[ns]
 2   event_name  440317 non-null  object        
 3   details     62740 non-null   float64       
dtypes: datetime64[ns](1), float64(1), object(2)
memory usage: 13.4+ MB

None

**************************************************

**************************************************

**************************************************

'Number of dublicate values'

0


dataset(marketing_events)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 14 entries, 0 to 13
Data columns (total 4 columns):
 #   Column     Non-Null Count  Dtype         
---  ------     --------------  -----         
 0   name       14 non-null     object        
 1   regions    14 non-null     object        
 2   start_dt   14 non-null     datetime64[ns]
 3   finish_dt  14 non-null     datetime64[ns]
dtypes: datetime64[ns](2), object(2)
memory usage: 576.0+ bytes

None

**************************************************

**************************************************

**************************************************

'Number of dublicate values'

0


dataset(new_users)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 61733 entries, 0 to 61732
Data columns (total 4 columns):
 #   Column      Non-Null Count  Dtype         
---  ------      --------------  -----         
 0   user_id     61733 non-null  object        
 1   first_date  61733 non-null  datetime64[ns]
 2   region      61733 non-null  object        
 3   device      61733 non-null  object        
dtypes: datetime64[ns](1), object(3)
memory usage: 1.9+ MB

None

**************************************************

**************************************************

**************************************************

'Number of dublicate values'

0


dataset(participants)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 18268 entries, 0 to 18267
Data columns (total 3 columns):
 #   Column   Non-Null Count  Dtype 
---  ------   --------------  ----- 
 0   user_id  18268 non-null  object
 1   group    18268 non-null  object
 2   ab_test  18268 non-null  object
dtypes: object(3)
memory usage: 428.3+ KB

None

**************************************************

**************************************************

**************************************************

'Number of dublicate values'

0


#let's check for which period the event data is provided
print('Date of the first event', events['event_dt'].min())
print('Data of the last event', events['event_dt'].max())

Date of the first event 2020-12-07 00:00:33
Data of the last event 2020-12-30 23:36:33


#let's check for what period the data is provided in the file with information about new users

print('The first date of registration', new_users['first_date'].min())
print('The last date of registration', new_users['first_date'].max())

The first date of registration 2020-12-07 00:00:00
The last date of registration 2020-12-23 00:00:00


#checking how many test there are in data
participants['ab_test'].unique()

array(['recommender_system_test', 'interface_eu_test'], dtype=object)


#let's split the dataframe with participants into 2 sets
recommender_system_test = participants.query('ab_test=="recommender_system_test"')
interface_eu_test = participants.query('ab_test=="interface_eu_test"')


#adding information about regions to the dataframe of test participants
recommender_system_test = pd.merge(new_users, recommender_system_test, how="inner", on='user_id')
recommender_system_test['first_date'].max()

Timestamp('2020-12-21 00:00:00')


#calculation of the share of users from the EU
share = len(recommender_system_test.query('region=="EU"')['user_id'])/len(new_users.query('region=="EU"')['user_id'])
print("{0:.2%}".format(share))

13.73%


count = round(len(new_users.query('region=="EU"')['user_id'])*0.15) 
nobs = len(new_users.query('region=="EU"')['user_id'])

stat, pval = proportions_ztest(count, nobs, share)
print('{0:0.3f}'.format(pval))

0.000


recommender_system_test.groupby(by=['group'])['user_id'].nunique()

group
A    3824
B    2877
Name: user_id, dtype: int64


print('Total number of test participants', recommender_system_test['user_id'].nunique())

Total number of test participants 6701


begin_date = pd.to_datetime("2020-12-07 00:00:00")
end_date = pd.to_datetime("2020-12-30 23:59:59")

temp = marketing_events[(marketing_events['start_dt'] >= begin_date) | (marketing_events['finish_dt'] >= end_date)]
temp


#let's divide the participants into 2 lists according to the tests
participants_recommender = recommender_system_test['user_id']
participants_interface = interface_eu_test['user_id']

#function for determining the intersection between lists
def intersection(l1, l2): 
    return list(set(l1) & set(l2)) 

print('The number of participants who got into both tests', len(intersection(participants_recommender, participants_interface)))
print('Share of the total number of participants', "{0:.2%}".format(
    len(intersection(participants_recommender, participants_interface))/len(recommender_system_test['user_id']))
     )

The number of participants who got into both tests 1602
Share of the total number of participants 23.91%


recommender_system_test_a = recommender_system_test.query('group == "A"')
recommender_system_test_b = recommender_system_test.query('group == "B"')

#let's divide the participants into 2 lists by test groups
participants_recommender_a = recommender_system_test_a['user_id']
participants_recommender_b = recommender_system_test_b['user_id']

print('The number of participants in both groups', len(intersection(participants_recommender_a, participants_recommender_b)))

The number of participants in both groups 0


fig = px.histogram(recommender_system_test, x="first_date",
             color='group', barmode='group',
             height=400, width=800)

fig.update_layout(barmode='stack', 
                  xaxis={'categoryorder':'total descending'}, 
                  title_text='Distribution of users by registration date between groups',
                  yaxis=dict(
                      title='Number of users', 
                      titlefont_size=16,
                      tickfont_size=14,
                  ))

fig.show()


fig = px.histogram(recommender_system_test, x="region",
             color='group', barmode='group',
             height=400, width=800)

fig.update_layout(barmode='stack', 
                  xaxis={'categoryorder':'total descending'}, 
                  title_text='Distribution of users in the test by region between groups',
                  yaxis=dict(
                      title='Number of users', 
                      titlefont_size=16,
                      tickfont_size=14,
                  ))

fig.show()


recommender_system_test = recommender_system_test.query('region=="EU"')


fig = px.histogram(recommender_system_test, x="device",
             color='group', barmode='group',
             height=400, width=800)

fig.update_layout(barmode='stack', 
                  xaxis={'categoryorder':'total descending'}, 
                  title_text='Distribution of users by device between groups',
                  yaxis=dict(
                      title='Number of users', 
                      titlefont_size=16,
                      tickfont_size=14,
                  ))

fig.show()


#let's add events from the events dataset to the dataset with test participants

events_test = pd.merge(recommender_system_test, events, how="inner", on='user_id')
events_test.head(5)


events_test['event_day_week'] = pd.DatetimeIndex(events_test['event_dt']).day_name()

week = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']
events_test['event_day_week'] = pd.Categorical(events_test['event_day_week'], categories=week, ordered=True)
events_test = events_test.sort_values('event_day_week')

fig = px.histogram(events_test, x="event_day_week",
             color='group', barmode='group',
             height=400, width=800)

fig.update_layout(barmode='stack', 
                  title_text='Distribution of events between groups by days of the week',
                  yaxis=dict(
                      title='Number of users', 
                      titlefont_size=16,
                      tickfont_size=14,
                  ))
fig.show()


#let's count the number of events by users

eventsByUsersA = events_test.query('group == "A"').groupby('user_id', as_index=False).agg({'event_dt': 'count'})
eventsByUsersB = events_test.query('group == "B"').groupby('user_id', as_index=False).agg({'event_dt': 'count'})

eventsByUsersA.columns = ['user_id', 'events']
eventsByUsersB.columns = ['user_id', 'events']

eventsByUsersA.head(5)


fig = plt.figure(figsize=(15,7))

x = eventsByUsersA['events']
y = eventsByUsersB['events']

bins = 30

plt.title('Distribution of the number of events per user in groups')
plt.grid(color='b', linestyle='-', linewidth=0.1)
plt.hist(x, bins, alpha=0.9, label='Group А')
plt.hist(y, bins, alpha=0.5, label='Group В')
plt.legend(loc='upper right')
plt.show()


sample_1 = eventsByUsersA['events']
sample_2 = eventsByUsersB['events']

alpha = .05

results = st.ttest_ind(
    sample_1, 
    sample_2)

print('p-value:', results.pvalue)

if results.pvalue < alpha:
    print("Reject the null hypothesis")
else:
    print("Cannot reject the null hypothesis")

p-value: 4.896419370297101e-16
Reject the null hypothesis


print('The average number of events per user in Group A:', sample_1.mean())
print('The average number of events per user in Group B:', sample_2.mean())
diff = (sample_1.mean() - sample_2.mean())/sample_1.mean()
print('Difference:', "{0:.2%}".format(diff))

The average number of events per user in Group A: 7.03110599078341
The average number of events per user in Group B: 5.827822120866591
Difference: 17.11%


fig = plt.figure(figsize=(15,7))

events_test.query('group == "A"')['event_dt'].hist(density=True, alpha=0.9, label="Группа А", bins=50)
events_test.query('group == "B"')['event_dt'].hist(density=True, bins=50, alpha=0.5, color='pink', label="Group В")

plt.title('Histogram of the number of observations by date')
plt.legend(loc="upper right")
plt.ylabel('Number of observations')
plt.show()


events_test_a = events_test.query('group == "A"')
events_test_b = events_test.query('group == "B"')

funnel_a = events_test_a.groupby('event_name').agg({'user_id':'nunique'})\
.sort_values(by='user_id', ascending=False).reset_index(level=[0,0])

#percentage of the total number of unique users
funnel_a['percent'] = funnel_a['user_id'] / events_test_a ['user_id'].nunique()
funnel_a['conversion'] = funnel_a['user_id']/funnel_a['user_id'].shift(periods=1)

#the first event in the funnel is 100%
funnel_a = funnel_a.fillna(1.00)

print('Funnel of events in Group A')

with pd.option_context('display.float_format', '{:.2%}'.format):
    display(funnel_a)

Funnel of events in Group A


#let's change the order of events so that the target action - purchase - is at the end
funnel_a = funnel_a.set_index('event_name')

new_index = ['login', 'product_page', 'product_cart', 'purchase']

funnel_a = funnel_a.reindex(new_index)
funnel_a = funnel_a.reset_index()

print('Funnel of events in Group A')

with pd.option_context('display.float_format', '{:.2%}'.format):
    display(funnel_a)

Funnel of events in Group A


# funnel chart

fig = go.Figure(
    go.Funnel(
        y = funnel_a['event_name'],
        x = funnel_a['user_id'],
    )
)
fig.update_layout(title_text='Event funnel for group A')
fig.show()


funnel_b = events_test_b.groupby('event_name').agg({'user_id':'nunique'})\
.sort_values(by='user_id', ascending=False).reset_index(level=[0,0])

#percentage of the total number of unique users
funnel_b['percent'] = funnel_b['user_id'] / events_test_b ['user_id'].nunique()
funnel_b['conversion'] = funnel_b['user_id']/funnel_b['user_id'].shift(periods=1)

#the first event in the funnel is 100%
funnel_b = funnel_b.fillna(1.00)

#let's change the order of events so that the target action - purchase - is at the end
funnel_b = funnel_b.set_index('event_name')

new_index = ['login', 'product_page', 'product_cart', 'purchase']

funnel_b = funnel_b.reindex(new_index)
funnel_b = funnel_b.reset_index()

print('Event funnel for group B')

with pd.option_context('display.float_format', '{:.2%}'.format):
    display(funnel_b)

Event funnel for group B


# funnel chart

fig = go.Figure(
    go.Funnel(
        y = funnel_b['event_name'],
        x = funnel_b['user_id'],
    )
)
fig.update_layout(title_text='Event funnel for group B')
fig.show()


#function for conducting the test

def test(successes1, successes2, trials1, trials2, alpha):
    
    p1 = successes1 / trials1
              
    p2 = successes2 / trials2
        
    # the proportion of success in the combined dataset:
    p_combined = (successes1 + successes2) / (trials2 + trials1)
    
    # the difference in proportions in datasets
    difference = p1 - p2
    
    # we count statistics in the standard deviations of the standard normal distribution
    z_value = difference / mth.sqrt(p_combined * (1 - p_combined) * (1/trials1 + 1/trials2))
    
    # we set the standard normal distribution (mean 0, standard deviation 1)
    distr = st.norm(0, 1) 
      
    p_value = (1 - distr.cdf(abs(z_value))) * 2
    print('p-value: ', p_value)
    
    if p_value < alpha:
        print('We reject the null hypothesis: there is a significant difference between the shares')       
    else:
        print('It was not possible to reject the null hypothesis, there is no reason to consider the shares are different')


users_total_a = events_test_a['user_id'].nunique()
users_total_b = events_test_b['user_id'].nunique()


events = ['product_page', 'product_cart', 'purchase']

funnel_a = funnel_a.set_index('event_name')
funnel_b = funnel_b.set_index('event_name')

for event in events:
    print('Test for the event:', event)
    test(funnel_a.loc[event, 'user_id'], funnel_b.loc[event, 'user_id'], users_total_a, users_total_b, 0.01)
    print(' '*50)

Test for the event: product_page
p-value:  6.942739359416805e-06
We reject the null hypothesis: there is a significant difference between the shares
                                                  
Test for the event: product_cart
p-value:  0.21469192029582396
It was not possible to reject the null hypothesis, there is no reason to consider the shares are different
                                                  
Test for the event: purchase
p-value:  0.04652482738393027
It was not possible to reject the null hypothesis, there is no reason to consider the shares are different

	details
count	62740.000000
mean	23.877631
std	72.180465
min	4.990000
25%	4.990000
50%	4.990000
75%	9.990000
max	499.990000

	user_id	event_dt	event_name	details
338089	C88FA2A19F927C1E	2020-12-18 06:31:38	login	NaN
243909	8C819478F2623906	2020-12-27 03:23:51	product_page	NaN
382124	1D6A02B0987CD71A	2020-12-22 01:09:55	login	NaN
70999	09FE8ABDED4AF737	2020-12-11 06:33:26	product_cart	NaN
405933	B578A3850EBC557E	2020-12-24 16:46:44	login	NaN
173103	C5B25E28ED032491	2020-12-16 23:18:35	product_page	NaN
260336	AB139261F9FEF6B5	2020-12-08 01:42:10	login	NaN
182583	87D6E4C509D5EBF1	2020-12-18 00:32:28	product_page	NaN
116462	7FDCDE6A910C6EA2	2020-12-25 08:38:33	product_cart	NaN
122570	E69F98E44CAA2913	2020-12-28 07:24:49	product_cart	NaN

	name	regions	start_dt	finish_dt
10	CIS New Year Gift Lottery	CIS	2020-12-30	2021-01-07
2	St. Patric's Day Promo	EU, N.America	2020-03-17	2020-03-19
9	Victory Day CIS (May 9th) Event	CIS	2020-05-09	2020-05-11
13	Chinese Moon Festival	APAC	2020-10-01	2020-10-07
4	4th of July Promo	N.America	2020-07-04	2020-07-11
5	Black Friday Ads Campaign	EU, CIS, APAC, N.America	2020-11-26	2020-12-01
3	Easter Promo	EU, CIS, APAC, N.America	2020-04-12	2020-04-19
11	Dragon Boat Festival Giveaway	APAC	2020-06-25	2020-07-01
6	Chinese New Year Promo	APAC	2020-01-25	2020-02-07
7	Labor day (May 1st) Ads Campaign	EU, CIS, APAC	2020-05-01	2020-05-03

	user_id	first_date	region	device
51998	58B7D471C59B2EF4	2020-12-19	APAC	Android
9813	37DD338227177554	2020-12-14	EU	PC
18527	5732FDB1C948FE26	2020-12-08	EU	Android
43292	2BA3E36AAFBDFD01	2020-12-18	EU	PC
8241	3A0B362A5E4BABC3	2020-12-14	EU	PC
16828	CDBE367CF4492D7E	2020-12-21	EU	Android
21446	4F79F4AFE44091C9	2020-12-15	CIS	Android
49791	775AF16EA3BDE170	2020-12-19	EU	Android
21794	DC1C43084F6156E5	2020-12-15	CIS	PC
54139	FB4C0EEC427DCB62	2020-12-13	APAC	Android

	user_id	group	ab_test
1213	A662A4DFC7F13881	A	recommender_system_test
13201	69C16A396D80FB81	B	interface_eu_test
2083	A35EA209D6186DB7	B	recommender_system_test
10808	5C530719759F0456	A	interface_eu_test
6775	0432FE839A9A354C	B	interface_eu_test
17923	9BA3280B1576B4E9	B	interface_eu_test
4594	B54431CE7FDB94E0	B	recommender_system_test
9142	1AE6FF352875CA8C	B	interface_eu_test
13988	20E8EBC0133C64EE	A	interface_eu_test
3991	107D9F8613EBF2AA	B	recommender_system_test

Table of Contents

A/B-testing¶

Data upload¶

Assessment of the correctness of the test¶

Checking the points of the terms of reference¶

Timing¶

User regions¶

Number of test participants¶

Time of the test¶

Audience¶

Distribution of users by registration date between groups¶

Distribution of participants by region between groups¶

Distribution of participants by device between groups¶

Distribution of events between groups by days of the week¶

Exploratory data analysis¶

Number of events per user in A/B samples¶

Distribution of the number of events in the samples by day¶

Funnels by groups¶

Conclusion on EDA and evaluation of the correctness of the test¶

Checking the statistical difference of the shares¶

General conclusion¶

	name	regions	start_dt	finish_dt
count	14	14	14	14
unique	14	6	14	14
top	St. Valentine's Day Giveaway	APAC	2020-07-04 00:00:00	2020-03-10 00:00:00
freq	1	4	1	1
first	NaN	NaN	2020-01-25 00:00:00	2020-02-07 00:00:00
last	NaN	NaN	2020-12-30 00:00:00	2021-01-07 00:00:00

	user_id	first_date	region	device
count	61733	61733	61733	61733
unique	61733	17	4	4
top	EDDF307171589F79	2020-12-21 00:00:00	EU	Android
freq	1	6290	46270	27520
first	NaN	2020-12-07 00:00:00	NaN	NaN
last	NaN	2020-12-23 00:00:00	NaN	NaN

	user_id	group	ab_test
count	18268	18268	18268
unique	16666	2	2
top	823E881A8B197337	A	interface_eu_test
freq	2	9655	11567

	name	regions	start_dt	finish_dt
0	Christmas&New Year Promo	EU, N.America	2020-12-25	2021-01-03
10	CIS New Year Gift Lottery	CIS	2020-12-30	2021-01-07

	user_id	first_date	region	device	group	ab_test	event_dt	event_name	details
0	D72A72121175D8BE	2020-12-07	EU	PC	A	recommender_system_test	2020-12-07 21:52:10	product_page	NaN
1	D72A72121175D8BE	2020-12-07	EU	PC	A	recommender_system_test	2020-12-07 21:52:07	login	NaN
2	DD4352CDCF8C3D57	2020-12-07	EU	Android	B	recommender_system_test	2020-12-07 15:32:54	product_page	NaN
3	DD4352CDCF8C3D57	2020-12-07	EU	Android	B	recommender_system_test	2020-12-08 08:29:31	product_page	NaN
4	DD4352CDCF8C3D57	2020-12-07	EU	Android	B	recommender_system_test	2020-12-10 18:18:27	product_page	NaN

	user_id	events
0	0010A1C096941592	12
1	00341D8401F0F665	2
2	003DF44D7589BBD4	15
3	00505E15A9D81546	5
4	006E3E4E232CE760	6

	event_name	user_id	percent	conversion
0	login	2604	100.00%	100.00%
1	product_page	1685	64.71%	64.71%
2	purchase	833	31.99%	49.44%
3	product_cart	782	30.03%	93.88%

	event_name	user_id	percent	conversion
0	login	877	100.00%	100.00%
1	product_page	493	56.21%	56.21%
2	product_cart	244	27.82%	97.99%
3	purchase	249	28.39%	50.51%